import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, GRU, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/GRU_12h_TFM'
TIME_STEPS=72 #12h
CMODEL = GRU
MODEL = "GRU"
UNITS=40
DROPOUT=0.405
ACTIVATION='tanh'
OPTIMIZER='adamax'
EPOCHS=68
BATCHSIZE=45
VALIDATIONSPLIT=0.1
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (3045, 72, 1) y_train shape: (3045,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(CMODEL(units = UNITS, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Dropout(rate=DROPOUT))
model.add(TimeDistributed(Dense(1,kernel_initializer='normal',activation=ACTIVATION)))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=['mse',rmse])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= gru (GRU) (None, 72, 40) 5160 _________________________________________________________________ dropout (Dropout) (None, 72, 40) 0 _________________________________________________________________ time_distributed (TimeDistri (None, 72, 1) 41 ================================================================= Total params: 5,201 Trainable params: 5,201 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/68 61/61 [==============================] - 2s 26ms/step - loss: 0.7880 - mse: 1.0116 - rmse: 0.7894 - val_loss: 0.8241 - val_mse: 0.7141 - val_rmse: 0.8244 Epoch 2/68 61/61 [==============================] - 1s 20ms/step - loss: 0.7017 - mse: 0.8416 - rmse: 0.7111 - val_loss: 0.5942 - val_mse: 0.3825 - val_rmse: 0.5980 Epoch 3/68 61/61 [==============================] - 1s 21ms/step - loss: 0.5983 - mse: 0.6855 - rmse: 0.6330 - val_loss: 0.2962 - val_mse: 0.1183 - val_rmse: 0.3210 Epoch 4/68 61/61 [==============================] - 1s 21ms/step - loss: 0.5729 - mse: 0.6444 - rmse: 0.6233 - val_loss: 0.2592 - val_mse: 0.0983 - val_rmse: 0.2890 Epoch 5/68 61/61 [==============================] - 1s 21ms/step - loss: 0.5708 - mse: 0.6372 - rmse: 0.6234 - val_loss: 0.2565 - val_mse: 0.0966 - val_rmse: 0.2857 Epoch 6/68 61/61 [==============================] - 2s 30ms/step - loss: 0.5676 - mse: 0.6318 - rmse: 0.6217 - val_loss: 0.2519 - val_mse: 0.0939 - val_rmse: 0.2809 Epoch 7/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5657 - mse: 0.6292 - rmse: 0.6210 - val_loss: 0.2474 - val_mse: 0.0914 - val_rmse: 0.2761 Epoch 8/68 61/61 [==============================] - 1s 21ms/step - loss: 0.5641 - mse: 0.6272 - rmse: 0.6206 - val_loss: 0.2446 - val_mse: 0.0897 - val_rmse: 0.2728 Epoch 9/68 61/61 [==============================] - 1s 20ms/step - loss: 0.5623 - mse: 0.6252 - rmse: 0.6196 - val_loss: 0.2405 - val_mse: 0.0874 - val_rmse: 0.2684 Epoch 10/68 61/61 [==============================] - 1s 22ms/step - loss: 0.5614 - mse: 0.6243 - rmse: 0.6196 - val_loss: 0.2374 - val_mse: 0.0856 - val_rmse: 0.2648 Epoch 11/68 61/61 [==============================] - 1s 21ms/step - loss: 0.5602 - mse: 0.6235 - rmse: 0.6193 - val_loss: 0.2345 - val_mse: 0.0840 - val_rmse: 0.2616 Epoch 12/68 61/61 [==============================] - 1s 22ms/step - loss: 0.5596 - mse: 0.6226 - rmse: 0.6192 - val_loss: 0.2325 - val_mse: 0.0828 - val_rmse: 0.2592 Epoch 13/68 61/61 [==============================] - 1s 23ms/step - loss: 0.5587 - mse: 0.6223 - rmse: 0.6192 - val_loss: 0.2304 - val_mse: 0.0817 - val_rmse: 0.2568 Epoch 14/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5580 - mse: 0.6218 - rmse: 0.6191 - val_loss: 0.2287 - val_mse: 0.0807 - val_rmse: 0.2547 Epoch 15/68 61/61 [==============================] - 1s 24ms/step - loss: 0.5576 - mse: 0.6217 - rmse: 0.6192 - val_loss: 0.2271 - val_mse: 0.0799 - val_rmse: 0.2528 Epoch 16/68 61/61 [==============================] - 2s 25ms/step - loss: 0.5570 - mse: 0.6208 - rmse: 0.6191 - val_loss: 0.2260 - val_mse: 0.0792 - val_rmse: 0.2514 Epoch 17/68 61/61 [==============================] - 1s 23ms/step - loss: 0.5562 - mse: 0.6204 - rmse: 0.6187 - val_loss: 0.2239 - val_mse: 0.0782 - val_rmse: 0.2491 Epoch 18/68 61/61 [==============================] - 1s 22ms/step - loss: 0.5560 - mse: 0.6202 - rmse: 0.6190 - val_loss: 0.2241 - val_mse: 0.0782 - val_rmse: 0.2489 Epoch 19/68 61/61 [==============================] - 1s 23ms/step - loss: 0.5555 - mse: 0.6199 - rmse: 0.6188 - val_loss: 0.2227 - val_mse: 0.0774 - val_rmse: 0.2472 Epoch 20/68 61/61 [==============================] - 2s 25ms/step - loss: 0.5552 - mse: 0.6198 - rmse: 0.6189 - val_loss: 0.2209 - val_mse: 0.0765 - val_rmse: 0.2453 Epoch 21/68 61/61 [==============================] - 1s 24ms/step - loss: 0.5547 - mse: 0.6193 - rmse: 0.6188 - val_loss: 0.2209 - val_mse: 0.0765 - val_rmse: 0.2450 Epoch 22/68 61/61 [==============================] - 1s 24ms/step - loss: 0.5546 - mse: 0.6196 - rmse: 0.6191 - val_loss: 0.2195 - val_mse: 0.0758 - val_rmse: 0.2434 Epoch 23/68 61/61 [==============================] - 1s 22ms/step - loss: 0.5544 - mse: 0.6193 - rmse: 0.6190 - val_loss: 0.2189 - val_mse: 0.0755 - val_rmse: 0.2426 Epoch 24/68 61/61 [==============================] - 1s 23ms/step - loss: 0.5540 - mse: 0.6189 - rmse: 0.6188 - val_loss: 0.2183 - val_mse: 0.0752 - val_rmse: 0.2419 Epoch 25/68 61/61 [==============================] - 1s 23ms/step - loss: 0.5538 - mse: 0.6188 - rmse: 0.6189 - val_loss: 0.2174 - val_mse: 0.0748 - val_rmse: 0.2408 Epoch 26/68 61/61 [==============================] - 1s 22ms/step - loss: 0.5537 - mse: 0.6187 - rmse: 0.6190 - val_loss: 0.2168 - val_mse: 0.0744 - val_rmse: 0.2400 Epoch 27/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5534 - mse: 0.6189 - rmse: 0.6190 - val_loss: 0.2163 - val_mse: 0.0742 - val_rmse: 0.2394 Epoch 28/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5532 - mse: 0.6188 - rmse: 0.6190 - val_loss: 0.2157 - val_mse: 0.0739 - val_rmse: 0.2386 Epoch 29/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5532 - mse: 0.6190 - rmse: 0.6191 - val_loss: 0.2152 - val_mse: 0.0736 - val_rmse: 0.2379 Epoch 30/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5529 - mse: 0.6186 - rmse: 0.6190 - val_loss: 0.2148 - val_mse: 0.0734 - val_rmse: 0.2374 Epoch 31/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5526 - mse: 0.6185 - rmse: 0.6189 - val_loss: 0.2146 - val_mse: 0.0733 - val_rmse: 0.2371 Epoch 32/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5526 - mse: 0.6183 - rmse: 0.6188 - val_loss: 0.2140 - val_mse: 0.0730 - val_rmse: 0.2363 Epoch 33/68 61/61 [==============================] - 2s 29ms/step - loss: 0.5525 - mse: 0.6186 - rmse: 0.6190 - val_loss: 0.2137 - val_mse: 0.0728 - val_rmse: 0.2359 Epoch 34/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5523 - mse: 0.6182 - rmse: 0.6189 - val_loss: 0.2135 - val_mse: 0.0727 - val_rmse: 0.2355 Epoch 35/68 61/61 [==============================] - 2s 25ms/step - loss: 0.5521 - mse: 0.6181 - rmse: 0.6187 - val_loss: 0.2134 - val_mse: 0.0727 - val_rmse: 0.2354 Epoch 36/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5518 - mse: 0.6177 - rmse: 0.6185 - val_loss: 0.2127 - val_mse: 0.0724 - val_rmse: 0.2346 Epoch 37/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5518 - mse: 0.6178 - rmse: 0.6186 - val_loss: 0.2123 - val_mse: 0.0722 - val_rmse: 0.2341 Epoch 38/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5514 - mse: 0.6174 - rmse: 0.6183 - val_loss: 0.2120 - val_mse: 0.0720 - val_rmse: 0.2337 Epoch 39/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5514 - mse: 0.6176 - rmse: 0.6184 - val_loss: 0.2118 - val_mse: 0.0719 - val_rmse: 0.2333 Epoch 40/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5513 - mse: 0.6175 - rmse: 0.6183 - val_loss: 0.2115 - val_mse: 0.0718 - val_rmse: 0.2329 Epoch 41/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5510 - mse: 0.6173 - rmse: 0.6181 - val_loss: 0.2116 - val_mse: 0.0718 - val_rmse: 0.2329 Epoch 42/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5511 - mse: 0.6172 - rmse: 0.6181 - val_loss: 0.2109 - val_mse: 0.0715 - val_rmse: 0.2321 Epoch 43/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5511 - mse: 0.6171 - rmse: 0.6183 - val_loss: 0.2103 - val_mse: 0.0712 - val_rmse: 0.2315 Epoch 44/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5511 - mse: 0.6171 - rmse: 0.6183 - val_loss: 0.2099 - val_mse: 0.0710 - val_rmse: 0.2310 Epoch 45/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5508 - mse: 0.6169 - rmse: 0.6181 - val_loss: 0.2099 - val_mse: 0.0710 - val_rmse: 0.2309 Epoch 46/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5507 - mse: 0.6168 - rmse: 0.6180 - val_loss: 0.2097 - val_mse: 0.0709 - val_rmse: 0.2306 Epoch 47/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5505 - mse: 0.6166 - rmse: 0.6178 - val_loss: 0.2093 - val_mse: 0.0707 - val_rmse: 0.2301 Epoch 48/68 61/61 [==============================] - 2s 29ms/step - loss: 0.5506 - mse: 0.6168 - rmse: 0.6179 - val_loss: 0.2089 - val_mse: 0.0705 - val_rmse: 0.2297 Epoch 49/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5504 - mse: 0.6165 - rmse: 0.6178 - val_loss: 0.2087 - val_mse: 0.0704 - val_rmse: 0.2293 Epoch 50/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5502 - mse: 0.6162 - rmse: 0.6176 - val_loss: 0.2086 - val_mse: 0.0704 - val_rmse: 0.2291 Epoch 51/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5500 - mse: 0.6160 - rmse: 0.6174 - val_loss: 0.2080 - val_mse: 0.0701 - val_rmse: 0.2285 Epoch 52/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5499 - mse: 0.6159 - rmse: 0.6172 - val_loss: 0.2080 - val_mse: 0.0701 - val_rmse: 0.2283 Epoch 53/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5499 - mse: 0.6158 - rmse: 0.6173 - val_loss: 0.2081 - val_mse: 0.0701 - val_rmse: 0.2284 Epoch 54/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5498 - mse: 0.6156 - rmse: 0.6171 - val_loss: 0.2077 - val_mse: 0.0699 - val_rmse: 0.2279 Epoch 55/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5497 - mse: 0.6159 - rmse: 0.6171 - val_loss: 0.2077 - val_mse: 0.0699 - val_rmse: 0.2278 Epoch 56/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5495 - mse: 0.6155 - rmse: 0.6168 - val_loss: 0.2071 - val_mse: 0.0697 - val_rmse: 0.2272 Epoch 57/68 61/61 [==============================] - 2s 29ms/step - loss: 0.5495 - mse: 0.6154 - rmse: 0.6169 - val_loss: 0.2073 - val_mse: 0.0697 - val_rmse: 0.2272 Epoch 58/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5493 - mse: 0.6150 - rmse: 0.6166 - val_loss: 0.2078 - val_mse: 0.0699 - val_rmse: 0.2275 Epoch 59/68 61/61 [==============================] - 2s 26ms/step - loss: 0.5492 - mse: 0.6151 - rmse: 0.6165 - val_loss: 0.2067 - val_mse: 0.0694 - val_rmse: 0.2264 Epoch 60/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5492 - mse: 0.6152 - rmse: 0.6165 - val_loss: 0.2066 - val_mse: 0.0694 - val_rmse: 0.2263 Epoch 61/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5492 - mse: 0.6151 - rmse: 0.6164 - val_loss: 0.2064 - val_mse: 0.0693 - val_rmse: 0.2260 Epoch 62/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5489 - mse: 0.6147 - rmse: 0.6163 - val_loss: 0.2054 - val_mse: 0.0689 - val_rmse: 0.2250 Epoch 63/68 61/61 [==============================] - 2s 28ms/step - loss: 0.5487 - mse: 0.6144 - rmse: 0.6160 - val_loss: 0.2051 - val_mse: 0.0688 - val_rmse: 0.2247 Epoch 64/68 61/61 [==============================] - 2s 29ms/step - loss: 0.5488 - mse: 0.6146 - rmse: 0.6161 - val_loss: 0.2055 - val_mse: 0.0689 - val_rmse: 0.2249 Epoch 65/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5487 - mse: 0.6143 - rmse: 0.6159 - val_loss: 0.2053 - val_mse: 0.0688 - val_rmse: 0.2246 Epoch 66/68 61/61 [==============================] - 2s 32ms/step - loss: 0.5484 - mse: 0.6140 - rmse: 0.6156 - val_loss: 0.2050 - val_mse: 0.0687 - val_rmse: 0.2243 Epoch 67/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5483 - mse: 0.6139 - rmse: 0.6155 - val_loss: 0.2047 - val_mse: 0.0686 - val_rmse: 0.2239 Epoch 68/68 61/61 [==============================] - 2s 27ms/step - loss: 0.5484 - mse: 0.6140 - rmse: 0.6156 - val_loss: 0.2044 - val_mse: 0.0684 - val_rmse: 0.2235
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,MODEL)
GRU: Mean Absolute Error: 0.2480 Root Mean Square Error: 0.4710 Mean Square Error: 0.2218
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.9 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,MODEL)
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ########################### Testing shape: (708, 72, 1)
<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
23/23 [==============================] - 0s 5ms/step - loss: 0.5961 - mse: 0.8772 - rmse: 0.6958 evaluate: [0.5961122512817383, 0.8772343993186951, 0.6957734227180481] GRU: Mean Absolute Error: 0.2153 Root Mean Square Error: 0.5445 Mean Square Error: 0.2965
anomalies: (300, 10)
###################################################### ####################### PM25 ########################### Testing shape: (708, 72, 1)
<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
23/23 [==============================] - 0s 5ms/step - loss: 0.6256 - mse: 0.9431 - rmse: 0.7251 evaluate: [0.6255766749382019, 0.9431400895118713, 0.7250567674636841] GRU: Mean Absolute Error: 0.2308 Root Mean Square Error: 0.5111 Mean Square Error: 0.2612
anomalies: (155, 10)
###################################################### ####################### PM10 ########################### Testing shape: (708, 72, 1)
<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
23/23 [==============================] - 0s 6ms/step - loss: 0.6460 - mse: 0.9883 - rmse: 0.7451 evaluate: [0.646030843257904, 0.98829185962677, 0.7450976371765137] GRU: Mean Absolute Error: 0.2419 Root Mean Square Error: 0.4842 Mean Square Error: 0.2345
anomalies: (107, 10)
###################################################### ####################### PM1ATM ########################### Testing shape: (708, 72, 1)
<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
23/23 [==============================] - 0s 5ms/step - loss: 0.6473 - mse: 0.9138 - rmse: 0.7525 evaluate: [0.6472610831260681, 0.913761556148529, 0.7524533867835999] GRU: Mean Absolute Error: 0.2418 Root Mean Square Error: 0.4945 Mean Square Error: 0.2445
anomalies: (148, 10)
###################################################### ####################### PM25ATM ########################### Testing shape: (708, 72, 1)
<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
23/23 [==============================] - 0s 5ms/step - loss: 0.6413 - mse: 0.8982 - rmse: 0.7453 evaluate: [0.6412870287895203, 0.8982148170471191, 0.7453376054763794] GRU: Mean Absolute Error: 0.2389 Root Mean Square Error: 0.5007 Mean Square Error: 0.2507
anomalies: (146, 10)
###################################################### ####################### PM10ATM ########################### Testing shape: (708, 72, 1)
<ipython-input-17-e1f1d6df3b5c>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
23/23 [==============================] - 0s 5ms/step - loss: 0.6347 - mse: 0.8948 - rmse: 0.7333 evaluate: [0.6347432732582092, 0.8947736620903015, 0.7333325743675232] GRU: Mean Absolute Error: 0.2396 Root Mean Square Error: 0.4993 Mean Square Error: 0.2493
anomalies: (148, 10)
######################################################